{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformers version:  2.3.0\n",
      "Tensorflow version:  2.1.0\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import tensorflow as tf\n",
    "import transformers\n",
    "from transformers import *\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import time\n",
    "import datetime\n",
    "import random\n",
    "\n",
    "print('Transformers version: ', transformers.__version__)\n",
    "print('Tensorflow version: ', tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    4342\n",
      "1    3271\n",
      "Name: target, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2644</th>\n",
       "      <td>3796</td>\n",
       "      <td>destruction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>So you have a new weapon that can cause un-ima...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2227</th>\n",
       "      <td>3185</td>\n",
       "      <td>deluge</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The f$&amp;amp;@ing things I do for #GISHWHES Just...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id      keyword location  \\\n",
       "2644  3796  destruction      NaN   \n",
       "2227  3185       deluge      NaN   \n",
       "\n",
       "                                                   text  target  \n",
       "2644  So you have a new weapon that can cause un-ima...       1  \n",
       "2227  The f$&amp;@ing things I do for #GISHWHES Just...       0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_dir = '/kaggle/input/nlp-getting-started/'\n",
    "train_df = pd.read_csv(data_dir+'train.csv')\n",
    "test_df = pd.read_csv(data_dir+'test.csv')\n",
    "train_df = train_df.sample(n=len(train_df), random_state=42)\n",
    "sample_submission = pd.read_csv(data_dir+'sample_submission.csv')\n",
    "print(train_df['target'].value_counts())\n",
    "train_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = train_df['text']\n",
    "y_train = train_df['target']\n",
    "x_test = test_df['text']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenization and Input Formatting\n",
    "### Sequence to IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode_tweet(df):\n",
    "    input_ids = []\n",
    "\n",
    "    for x in df:\n",
    "        encoded_x = tokenizer.encode(x,\n",
    "                                    add_special_tokens = True)\n",
    "        input_ids.append(encoded_x)\n",
    "    return input_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original:  Just happened a terrible car crash\n",
      "Encoded:  [101, 2074, 3047, 1037, 6659, 2482, 5823, 102]\n"
     ]
    }
   ],
   "source": [
    "test_input = encode_tweet(x_test)\n",
    "print('Original: ', x_test[0])\n",
    "print('Encoded: ', test_input[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input = encode_tweet(x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max sentence length in Train Ids:  84\n",
      "Max sentence length in Test Ids:  73\n"
     ]
    }
   ],
   "source": [
    "print('Max sentence length in Train Ids: ', max([len(sen) for sen in train_input]))\n",
    "print('Max sentence length in Test Ids: ', max([len(sen) for sen in test_input]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "MAX_LEN = 84\n",
    "\n",
    "def pad_tweets(df):\n",
    "    df = pad_sequences(df, maxlen=MAX_LEN, dtype=\"long\", \n",
    "                       value=0, truncating=\"post\", padding=\"post\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input = pad_tweets(train_input)\n",
    "test_input = pad_tweets(test_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Attention Masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_att_mask(df):\n",
    "    attention_masks = []\n",
    "\n",
    "    for tweet in df:\n",
    "        att_mask = [int(token_id > 0) for token_id in tweet]\n",
    "        attention_masks.append(att_mask)\n",
    "    return attention_masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_att = get_att_mask(train_input)\n",
    "test_att = get_att_mask(test_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "tr_input, val_input, tr_label, val_label = train_test_split(train_input, y_train, \n",
    "                                                            random_state=2020, test_size=0.15)\n",
    "# Do the same for the masks.\n",
    "tr_mask, val_mask, _, _ = train_test_split(train_att, y_train,\n",
    "                                             random_state=2020, test_size=0.15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting to Torch Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Training and Validation data and masks\n",
    "tr_input = torch.tensor(tr_input)\n",
    "val_input = torch.tensor(val_input)\n",
    "\n",
    "#convert to np.array, otherwise throws a mysterious 'KeyError: 4' error\n",
    "tr_label = torch.tensor(np.array(tr_label))\n",
    "val_label = torch.tensor(np.array(val_label))\n",
    "\n",
    "tr_mask = torch.tensor(tr_mask)\n",
    "val_mask = torch.tensor(val_mask)\n",
    "\n",
    "# For Test data and mask\n",
    "te_input = torch.tensor(test_input)\n",
    "te_mask = torch.tensor(test_att)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using helper classes in order to use batches for training. It creates an iterator, which should save on memory during training. The same must be repeated on the test set once we have prediction labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "BATCH_SIZE = 32 \n",
    "\n",
    "# For training\n",
    "train_data = TensorDataset(tr_input, tr_mask, tr_label)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler = train_sampler, batch_size = BATCH_SIZE)\n",
    "\n",
    "# For validation\n",
    "val_data = TensorDataset(val_input, val_mask, val_label)\n",
    "val_sampler = RandomSampler(val_data)\n",
    "val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size = BATCH_SIZE)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the Model\n",
    "Using the pre-trained model, documentation can be found here\n",
    "https://huggingface.co/transformers/v2.2.0/main_classes/model.html#transformers.PreTrainedModel.from_pretrained"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
    "\n",
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\",\n",
    "                                                      num_labels = 2,\n",
    "                                                      output_attentions = False,\n",
    "                                                      output_hidden_states = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Displaying some of the model's parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The BERT model has 201 different named parameters.\n",
      "\n",
      "==== Embedding Layer ====\n",
      "\n",
      "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
      "bert.embeddings.position_embeddings.weight                (512, 768)\n",
      "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
      "bert.embeddings.LayerNorm.weight                              (768,)\n",
      "bert.embeddings.LayerNorm.bias                                (768,)\n",
      "\n",
      "==== First Transformer ====\n",
      "\n",
      "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
      "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
      "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
      "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
      "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
      "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
      "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
      "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
      "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
      "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
      "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
      "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
      "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
      "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
      "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
      "\n",
      "==== Output Layer ====\n",
      "\n",
      "bert.pooler.dense.weight                                  (768, 768)\n",
      "bert.pooler.dense.bias                                        (768,)\n",
      "classifier.weight                                           (2, 768)\n",
      "classifier.bias                                                 (2,)\n"
     ]
    }
   ],
   "source": [
    "# Get all of the model's parameters as a list of tuples.\n",
    "params = list(model.named_parameters())\n",
    "\n",
    "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
    "\n",
    "print('==== Embedding Layer ====\\n')\n",
    "\n",
    "for p in params[0:5]:\n",
    "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
    "\n",
    "print('\\n==== First Transformer ====\\n')\n",
    "\n",
    "for p in params[5:21]:\n",
    "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
    "\n",
    "print('\\n==== Output Layer ====\\n')\n",
    "\n",
    "for p in params[-4:]:\n",
    "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AdamW optimizer\n",
    "\n",
    "https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L109"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import get_linear_schedule_with_warmup\n",
    "\n",
    "# AdamW is a class from the Huggingface library\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = 2e-5, # default is 5e-5\n",
    "                  eps = 1e-8 # default is 1e-8\n",
    "                )\n",
    "epochs = 1 # 1 epoch gave the best result\n",
    "\n",
    "# the number of batches times the number of epochs\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "\n",
    "# the learning rate scheduler\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0,\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Accuracy helper function\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_time(elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1 GPU(s) available.\n",
      "We will use the GPU: Tesla P100-PCIE-16GB\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():    \n",
    "\n",
    "    # Tell PyTorch to use the GPU.    \n",
    "    device = torch.device(\"cuda\")\n",
    "\n",
    "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
    "\n",
    "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
    "\n",
    "# If not...\n",
    "else:\n",
    "    print('No GPU available, using the CPU instead.')\n",
    "    device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-08T16:25:47.645884Z",
     "start_time": "2020-03-08T16:25:47.589131Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'random' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m--------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-45a36478cf60>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mseed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'random' is not defined"
     ]
    }
   ],
   "source": [
    "seed = 50\n",
    "\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "loss_arr = []\n",
    "\n",
    "for i in range(0, epochs):\n",
    "    \n",
    "    # ========= Training ==========\n",
    "    \n",
    "    print('====== Epoch {:} of {:}'.format(i+1, epochs))\n",
    "    print('Training...')\n",
    "    \n",
    "    t0 = time.time()\n",
    "    \n",
    "    total_loss = 0\n",
    "    # initialize training mode\n",
    "    model.train()\n",
    "    \n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        if step % 30 == 0 and not step == 0:\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            print('Batch {:>5,} of {:>5,}. Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "            \n",
    "            # Unpacking the training batch from dataloader and copying each tensor to the GPU\n",
    "            \n",
    "        b_input_ids = batch[0].long().to(device)\n",
    "        b_input_mask = batch[1].long().to(device)\n",
    "        b_labels = batch[2].long().to(device)\n",
    "        \n",
    "        # pytorch doesn't clear previously calculated gradients\n",
    "        # before performing backward pass, so clearing here:\n",
    "        model.zero_grad()\n",
    "        \n",
    "        outputs = model(b_input_ids,\n",
    "                       token_type_ids = None, \n",
    "                       attention_mask = b_input_mask,\n",
    "                       labels = b_labels)\n",
    "        loss = outputs[0]\n",
    "        \n",
    "        total_loss += loss.item()\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        #update the learning rate\n",
    "        scheduler.step()\n",
    "    \n",
    "    avg_train_loss = total_loss / len(train_dataloader)\n",
    "    \n",
    "    loss_arr.append(avg_train_loss)\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epoch took: {:}\".format(format_time(time.time() - t0)))\n",
    "    \n",
    "    # ========= Validation ==========\n",
    "    \n",
    "    print(\"\")\n",
    "    print(\"Running Validation...\")\n",
    "    t0 = time.time()\n",
    "    # evaluation mode\n",
    "    model.eval()\n",
    "    \n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    \n",
    "    for batch in val_dataloader:\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            \n",
    "            outputs = model(b_input_ids, \n",
    "                           token_type_ids = None, \n",
    "                           attention_mask = b_input_mask)\n",
    "            \n",
    "        logits = outputs[0]\n",
    "        # move logits to cpu\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        # get accuracy\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "        \n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "        \n",
    "        nb_eval_steps += 1\n",
    "    \n",
    "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
    "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
    "    \n",
    "print(\"\")\n",
    "print(\"Training complete!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvYAAAGXCAYAAADYoRRCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X9clfX9//HnQURTUQRRLMysE0dN0aYMzR8UgjhXqagff/DDmrOs0LKsto+pWx8oraabMsLV9hlWU5k/cKEtFc1MHJrLZmmCYFO3ZQoeAUUwz/n+0YfzjfghB4/n6MXjfrt52+28r/f7ut4Xr5xP3ryvC5PdbrcLAAAAwA3Ny9MTAAAAAHD1CPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AmqnXXntNFotFp0+fbtL4yspKWSwWLViwwMUzc86qVatksVh04MABj84DADzN29MTAIDmzGKxNLpvTk6OgoODr+FsAAA3MoI9AHjQK6+8UuPz/v37tWbNGk2aNEkDBgyocczf39+l137qqac0a9YstWrVqknjW7VqpX/84x9q0aKFS+cFAGgagj0AeNCYMWNqfL58+bLWrFmj/v371zpWH7vdroqKCrVp08apa3t7e8vb++r+GWjqNwUAANdjjz0A3EA+/PBDWSwWZWdnKyMjQ6NGjVLfvn319ttvS5L+/ve/67nnntPIkSPVr18//eAHP1BcXJx27NhR61x17bGvbjtx4oQWL16sYcOGqW/fvho3bpx2795dY3xde+y/27Zv3z5NmTJF/fr106BBg7RgwQJVVFTUmkdubq4mTpyovn37aujQoVq8eLEOHToki8Wi3/3ud03+Wp05c0YLFizQ8OHD1adPH913331KTk7WuXPnavS7cOGCli5dqpiYGIWGhiosLEwPPPCAli5dWqPftm3bNGXKFIWHhys0NFT33XefZs+erRMnTjR5jgDgSqzYA8AN6I033lBZWZnGjx+vgIAAdevWTZL017/+VcePH9fo0aN18803q6SkRBs2bNDMmTO1fPlyjRw5slHnf+aZZ9SqVSv99Kc/VWVlpf74xz/qscce09atW9WlS5crjj948KDef/99TZgwQQ8++KD27NmjNWvWyMfHRy+88IKj3549ezRjxgz5+/vr0UcfVbt27bRp0ybl5eU17Qvzf6xWqyZNmqR///vfmjhxonr27KmDBw/q7bffVl5enjIzM3XTTTdJkubPn69NmzZp3Lhx6t+/vy5duqQvv/xSf/vb3xzn++ijj5SUlKTevXtr5syZateunU6dOqXdu3fr5MmTjq8/AHgSwR4AbkBff/213nvvPfn5+dVof+qpp2ptyUlISNCDDz6o119/vdHBvkuXLlq2bJlMJpMkOVb+//znPyspKemK448cOaK1a9eqd+/ekqQpU6Zo2rRpWrNmjZ577jn5+PhIkl5++WW1bNlSmZmZ6tq1qyRp6tSpmjx5cqPmWZ/09HSdPHlSKSkpmjBhgqP9zjvv1OLFix3fqNjtdm3fvl1RUVF6+eWX6z3ftm3bJEkZGRny9fV1tDfmawEA7sJWHAC4AY0fP75WqJdUI9RXVFTo7Nmzqqys1A9/+EMdPnxYVVVVjTr/tGnTHKFekgYMGKCWLVvqyy+/bNT4sLAwR6ivNmjQIFVVVek///mPJOlf//qXjhw5opiYGEeolyQfHx8lJiY26jr1qf7JQmxsbI32+Ph4+fr6auvWrZIkk8mktm3b6siRIyosLKz3fL6+vrLb7Xr//fd1+fLlq5obAFwrrNgDwA3otttuq7P966+/1tKlS7Vjxw6dPXu21vGysjIFBARc8fzf31piMpnUoUMHWa3WRs2vrq0p1d+IWK1Wde/eXSdPnpQk9ejRo1bfutoay26369///rcGDRokL6+a61c+Pj669dZbHdeWpHnz5um///u/NXr0aHXv3l3h4eGKjIzUvffe6/jmZtq0afrggw80b948LVq0SAMHDtSwYcM0evRodezYsclzBQBXItgDwA2oen/4d12+fFkPPfSQTp48qcTERN11113y9fWVl5eXVq9erffff182m61R5/9+IK5mt9uvavx3z9HYcznL2fP+6Ec/Unh4uD788EPt3btXH330kTIzMzV48GC9+eab8vb2VqdOnbRhwwbt27dPubm52rdvn5KTk7Vs2TL9/ve/V58+fa7JvQCAMwj2AGAQn332mQoLC/X000/r0UcfrXGs+q0515PqX7Z17NixWsfqamssLy8v3XLLLSoqKpLNZqvxTUZVVZWOHz+uW2+9tcYYf39/jR07VmPHjpXdbtdLL72klStX6sMPP1RkZKSkb18POnjwYA0ePFjSt1/vCRMmaMWKFVq+fHmT5wsArsIeewAwiOoA+/0V688//1w7d+70xJQaFBwcrJCQEL3//vuOfffSt+F75cqVV3XuqKgoffXVV8rKyqrR/qc//UllZWWKjo6WJF26dEnl5eU1+phMJvXq1UuSHK/GLCkpqXUNs9ksHx+fRm9PAoBrjRV7ADAIi8Wi2267Ta+//rpKS0t12223qbCwUJmZmbJYLPr88889PcVafvazn2nGjBn6r//6L02ePFlt27bVpk2bajy42xQzZ87Uli1b9MILL+jTTz+VxWLRZ599pvXr1yskJEQPPfSQpG/3+0dFRSkqKkoWi0X+/v46ceKEVq1apY4dOyoiIkKS9Nxzz6m0tFSDBw/WLbfcogsXLig7O1uVlZUaO3bs1X4ZAMAlCPYAYBA+Pj5644039Morr2jdunWqrKxUSEiIlixZov3791+XwX7IkCFasWKFfv3rXys9PV0dOnTQ/fffr6ioKMXFxal169ZNOq+fn5/WrFmj5cuXKycnR+vWrVNAQIDi4+M1a9YsxzMKvr6+io+P1549e7Rr1y5VVFQoMDBQI0eO1KOPPip/f39JUmxsrDZu3Kj169fr7Nmz8vX11Z133qm0tDSNGDHCZV8PALgaJvu1enoJAIAm+stf/qJnn31Wv/3tbxUVFeXp6QDADYE99gAAj7HZbLXerV9VVaWMjAz5+PhowIABHpoZANx42IoDAPCY8vJyjR49Wg888IBuu+02lZSUaNOmTSooKFBSUhLviAcAJxDsAQAe07p1aw0ZMkRbtmzRmTNnJEm33367XnzxRU2aNMnDswOAGwt77AEAAAADYI89AAAAYAAEewAAAMAA2GPvpLNnz8tmY/eSOwQEtFNxcfmVO+KGRp2Njxo3D9S5eaDO7uHlZVLHjm2dHkewd5LNZifYuxFf6+aBOhsfNW4eqHPzQJ2vX2zFAQAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABuDRYH/+/HklJydr6NChCg0NVWxsrHJycpw6h91uV2JioiwWi1JSUmodP336tH75y19qxIgRCg0NVWRkpBYsWKBTp0656jYAAAAAj/P25MWTkpJ06NAhzZ07V8HBwdqwYYOSkpKUnp6uiIiIRp0jMzNTRUVFdR6rqqpSfHy8zp07p9mzZ+uOO+5QYWGhli1bpr/97W/Kzs6Wj4+PK28JAAAA8AiPBfudO3cqNzdXqampio6OliQNGjRIJ06c0KJFixoV7E+dOqVXX31VKSkpmj17dq3jn3zyib788kslJydr4sSJkqTw8HC1bNlSL7zwgj755BOFh4e79sYAAAAAD/DYVpytW7fK19dXI0aMcLSZTCaNGzdORUVFOnr06BXPsXDhQg0cOFAxMTF1Hvf2/vb7Fl9f3xrt1Z9ZrQcAAIBReGzFvqCgQGazWV5eNb+3sFgskqT8/HyZzeZ6x2dnZysvL0+bN2+ut0///v0VGhqq1NRU3XLLLbr99ttVVFSk1NRUhYWFqV+/fq65GQAAAMDDPLZib7Va1aFDh1rt1W1Wq7XesSUlJUpJSdGcOXPUtWvXevu1aNFCf/zjH9W9e3dNmDBBP/jBDzRhwgQFBQVpxYoVtb6pAAAAAG5UHn141mQyNelYSkqKgoODFR8f3+D5L126pGeeeUYFBQV66aWX1L17dxUWFio1NVWPP/643nzzTbVs2dKpOQcEtHOqP65OYKDvlTvhhkedjY8aNw/UuXmgztcvjwV7Pz+/Olflz507J0l1ruZL0u7du7V582ZlZGSovLy8xrGqqiqVlpaqTZs28vb21rp167Rjxw5t3LhRPXv2lCQNHDhQPXr0UEJCgjZt2qSxY8c6Ne/i4nLZbHanxqBpAgN9dfp0maengWuMOhsfNW4eqHPzQJ3dw8vL1KTFZI/tRTGbzSosLJTNZqvRnp+fL0kKCQmpc1xBQYFsNpsSEhIUFhbm+CNJq1evVlhYmHJzcyVJhw4dUsuWLR2hvlqfPn0kqVEP6AIAAAA3Ao+t2EdHR2vt2rXavn27oqKiHO1ZWVnq0aNHvQ/Ojho1Sr169arVnpiYqJiYGMXFxTkewO3cubMuXbqkQ4cOqXfv3o6+Bw4ckCR16dLFlbcEAAAAeIzHgn1ERITCw8M1b948Wa1WBQcHKysrS/v371daWpqjX0JCgvbu3asjR45IkoKCghQUFFTnObt06VLjvfSxsbH64x//qKSkJD322GPq1q2bCgsLlZaWpk6dOun++++/tjcJAAAAuInHgr3JZFJaWpqWLFmipUuXqrS0VGazWampqYqMjHTJNW6++Wb9+c9/Vmpqql5//XWdOXNGgYGBioiIUFJSkjp27OiS6wAAAACeZrLb7TwJ6gQennUfHtBpHqiz8VHj5oE6Nw/U2T1uuIdnAQAAALgOwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAeDfbnz59XcnKyhg4dqtDQUMXGxionJ8epc9jtdiUmJspisSglJaXGsfXr18tisdT7Z9OmTa68HQAAAMBjvD158aSkJB06dEhz585VcHCwNmzYoKSkJKWnpysiIqJR58jMzFRRUVGdx+69916tWbOmVntKSoqOHDmiYcOGXdX8AQAAgOuFx4L9zp07lZubq9TUVEVHR0uSBg0apBMnTmjRokWNCvanTp3Sq6++qpSUFM2ePbvWcX9/f/n7+9doKy4u1uHDhxUTE6P27du75mYAAAAAD/PYVpytW7fK19dXI0aMcLSZTCaNGzdORUVFOnr06BXPsXDhQg0cOFAxMTGNvm5WVpYuXbqkCRMmNGneAAAAwPXIYyv2BQUFMpvN8vKq+b2FxWKRJOXn58tsNtc7Pjs7W3l5edq8ebNT112/fr1uueUWDRo0yPlJAwAAANcpj63YW61WdejQoVZ7dZvVaq13bElJiVJSUjRnzhx17dq10dc8cOCAjh49qtjYWJlMJucnDQAAAFynPPrwbEPhuqFjKSkpCg4OVnx8vFPXW7dunby8vBQbG+vUuO8KCGjX5LFwXmCgr6enADegzsZHjZsH6tw8UOfrl8eCvZ+fX52r8ufOnZOkOlfzJWn37t3avHmzMjIyVF5eXuNYVVWVSktL1aZNG3l717y1iooKbd68WYMHD9bNN9/c5HkXF5fLZrM3eTwaLzDQV6dPl3l6GrjGqLPxUePmgTo3D9TZPby8TE1aTPbYVhyz2azCwkLZbLYa7fn5+ZKkkJCQOscVFBTIZrMpISFBYWFhjj+StHr1aoWFhSk3N7fWuPfff1/l5eU8NAsAAABD8tiKfXR0tNauXavt27crKirK0Z6VlaUePXrU++DsqFGj1KtXr1rtiYmJiomJUVxcnOMB3O9at26d/Pz8alwLAAAAMAqPBfuIiAiFh4dr3rx5slqtCg4OVlZWlvbv36+0tDRHv4SEBO3du1dHjhyRJAUFBSkoKKjOc3bp0kXh4eG12k+cOKF9+/YpLi5OPj4+1+aGAAAAAA/yWLA3mUxKS0vTkiVLtHTpUpWWlspsNis1NVWRkZEuvda6detkt9s1fvx4l54XAAAAuF6Y7HY7T4I6gYdn3YcHdJoH6mx81Lh5oM7NA3V2jxvu4VkAAAAArkOwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYgNPB/p///Kc+/PDDGm2ffvqpZs6cqcmTJ2vNmjUumxwAAACAxvF2dsBrr70mq9Wq4cOHS5JKSko0Y8YMXbhwQa1atdIvfvELBQQEKCoqyuWTBQAAAFA3p1fsP/vsM91zzz2Oz5s2bVJ5ebnWr1+vPXv2qF+/fsrIyHDpJAEAAAA0zOlgX1JSos6dOzs+79q1Sz/4wQ8UEhIiHx8fjR49WoWFhS6dJAAAAICGOR3sb7rpJpWVlUmSLl++rP3792vgwIGO461bt1Z5ebnrZggAAADgipwO9nfeeac2btyos2fPKjMzUxcuXNCQIUMcx//1r3/J39/fpZMEAAAA0DCnH56dPn26Hn/8ccc++169etVYsd+9e7d69+7tuhkCAAAAuCKng/29996rjIwM5eTkqF27doqPj5fJZJIknT17VkFBQRo7dqzLJwoAAACgfia73W739CRuJMXF5bLZ+JK5Q2Cgr06fLvP0NHCNUWfjo8bNA3VuHqize3h5mRQQ0M7pcU6v2Nflm2++UU5Ojs6dO6f77rtPgYGBrjgtAAAAgEZyOti/8sorysvL07p16yRJdrtdDz/8sD7++GPZ7Xb5+fkpMzNTt956q8snCwAAAKBuTr8VZ9euXTUelt2+fbv27dun6dOn61e/+pUk6Xe/+53rZggAAADgipxesf/qq6/UvXt3x+cdO3YoODhYc+fOlSQVFBTo3Xffdd0MAQAAAFyR08H+0qVLatGiheNzXl6e49WXktStWzedPn3aNbMDANyw9nz+ldbvLFRJaaX827dSbMQdGnxXkKenBQCG5fRWnKCgIB04cEDSt6vzJ06cUFhYmON4cXGx2rRp47oZAgBuOHs+/0oZ732h4tJK2SUVl1Yq470vtOfzrzw9NQAwLKdX7H/84x8rLS1NJSUlKigoULt27RQREeE4fvjwYR6cBYBmbv3OQlV9Y6vRVvWNTet3FrJqDwDXiNMr9o8++qjGjRunAwcOyGQyafHixWrfvr0kqaysTNu3b9fgwYNdPlEAwI2juLTSqXYAwNVzesXex8dHL730Up3H2rZtq48++kitW7e+6okBAG5cAe1b1RniA9q38sBsAKB5cHrFvsGTeXnJ19dXLVu2dOVpAQA3mNiIO+TjXfOfGB9vL8VG3OGhGQGA8TXpN89euHBBb775prZu3aqTJ09KkoKDgzVy5EhNnz6dh2cBoJmr3kfPW3EAwH1Mdrvd7swAq9WquLg4FRYWqmPHjurRo4ck6csvv1RJSYnuuOMOvfPOO/Lz87smE/a04uJy2WxOfcnQRIGBvjp9uszT08A1Rp2Njxo3D9S5eaDO7uHlZVJAQDunxzm9Yr9s2TIVFRVp/vz5mjx5suOd9pcvX9aaNWuUnJys1NRUvfDCC05PBgAAAEDTOL3Hfvv27Zo4caLi4uJq/KKqFi1aaOrUqRo/fry2bdvm0kkCAAAAaJjTwf7MmTPq1atXvcd79+6tM2fOXNWkAAAAADjH6WDfqVMnHT58uN7jhw8fVqdOnRp1rvPnzys5OVlDhw5VaGioYmNjlZOT49R87Ha7EhMTZbFYlJKSUmefEydO6Nlnn9WQIUPUp08f3XffffrFL37h1HUAAACA65nTwf6+++7T2rVrtXr1atls//+3CtpsNq1Zs0br1q1TZGRko86VlJSkd999V08++aRWrFghs9mspKQk7dy5s9HzyczMVFFRUb3Hv/jiC40fP17FxcWaP3++/vCHP+jJJ59Uq1a8SxkAAADG4fRbcc6ePavJkyfr+PHj8vf3d7wV59ixYyopKdGtt96q1atXq2PHjg2eZ+fOnXrkkUeUmpqq6OhoSd+uvk+dOlVWq1XvvffeFedy6tQp/fjHP1ZKSopmz56txMREzZs3z3HcbrfrwQcf1M0336z09HSZTCZnbrVOvBXHfXjyvnmgzsZHjZsH6tw8UGf3aOpbcZxese/YsaPWrVunRx55RH5+fjp48KAOHjyojh076pFHHtG6deuuGOolaevWrfL19dWIESMcbSaTSePGjVNRUZGOHj16xXMsXLhQAwcOVExMTJ3H9+7dq/z8fE2fPt0loR4AAAC4XjXpF1S1a9dOc+bM0Zw5c2odW716tVauXKnNmzc3eI6CggKZzWZ5edX83sJisUiS8vPzZTab6x2fnZ2tvLy8Bq+zb98+Sd9uE5oyZYoOHjyom266ScOGDdPzzz+vLl26NDhHAAAA4Ebh9Ir9lZw9e1bHjh27Yj+r1aoOHTrUaq9us1qt9Y4tKSlRSkqK5syZo65du9bb7+uvv5YkzZo1S3fffbfefPNNPfvss8rNzVVCQoIqKiquOE8AAADgRtCkFXtXaWh7TEPHUlJSFBwcrPj4+AbPX/34wI9+9CM999xzkqRBgwapc+fOevTRR5Wdna2JEyc6Neem7HdC0wUG+np6CnAD6mx81Lh5oM7NA3W+fnks2Pv5+dW5Kn/u3DlJqnM1X5J2796tzZs3KyMjQ+Xl5TWOVVVVqbS0VG3atJG3t7f8/PwkScOGDavRb8iQIWrRooU+//xzp4M9D8+6Dw/oNA/U2fiocfNAnZsH6uwebnt41lXMZrMKCwtrvDJT+nZvvSSFhITUOa6goEA2m00JCQkKCwtz/JG+3d8fFham3NzcBs9R7fv7+wEAAIAblcdW7KOjo7V27Vpt375dUVFRjvasrCz16NGj3gdnR40aVedvvk1MTFRMTIzi4uIcD+AOHz5crVu31s6dOx2v1JSkXbt26fLlywoNDXXxXQEAAACe0ahg/7//+7+NPuHf//73RvWLiIhQeHi45s2bJ6vVquDgYGVlZWn//v1KS0tz9EtISNDevXt15MgRSVJQUJCCgoLqPGeXLl0UHh7u+NyhQwc98cQTWrp0qdq1a6fhw4fryy+/1G9+8xv17NlTo0ePbvR9AQAAANezRgX7xYsXO3XSxrwz3mQyKS0tTUuWLNHSpUtVWloqs9ms1NTURv/m2sZ45JFH5Ovrq7feektvv/222rdvr5EjR+qZZ56Rj4+Py64DAAAAeFKjfvPs3r17nT7xD3/4wyZN6HrHw7PuwwM6zQN1Nj5q3DxQ5+aBOrtHUx+ebdSKvVFDOgAAAGAUvBYGAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAADwa7M+fP6/k5GQNHTpUoaGhio2NVU5OjlPnsNvtSkxMlMViUUpKSq3jFoulzj+rVq1y1W0AAAAAHuftyYsnJSXp0KFDmjt3roKDg7VhwwYlJSUpPT1dERERjTpHZmamioqKGuwzevRoTZs2rUZbt27dmjxvAAAA4HrjsWC/c+dO5ebmKjU1VdHR0ZKkQYMG6cSJE1q0aFGjgv2pU6f06quvKiUlRbNnz663X6dOndS/f3+XzR0AAAC43nhsK87WrVvl6+urESNGONpMJpPGjRunoqIiHT169IrnWLhwoQYOHKiYmJhrOVUAAADguuexYF9QUCCz2Swvr5pTsFgskqT8/PwGx2dnZysvL08LFy684rU2btyo0NBQ9e3bVxMnTtTmzZubPnEAAADgOuSxrThWq1W33XZbrfYOHTo4jtenpKREKSkpmjNnjrp27drgdR544AFFRESoa9eu+vrrr7WN7MegAAAW1ElEQVRq1SrNmTNHp0+frrXvHgAAALhRefThWZPJ1KRjKSkpCg4OVnx8/BWv8dprr9X4PGrUKCUkJOjXv/61Jk2apNatWzd+wpICAto51R9XJzDQ19NTgBtQZ+Ojxs0DdW4eqPP1y2PB3s/Pr85V+XPnzkn6/yv337d7925t3rxZGRkZKi8vr3GsqqpKpaWlatOmjby96741Ly8vPfjgg/r444+Vn5+v0NBQp+ZdXFwum83u1Bg0TWCgr06fLvP0NHCNUWfjo8bNA3VuHqize3h5mZq0mOyxPfZms1mFhYWy2Ww12qv31oeEhNQ5rqCgQDabTQkJCQoLC3P8kaTVq1crLCxMubm5DV67+prf398PAAAA3Kg8tmIfHR2ttWvXavv27YqKinK0Z2VlqUePHjKbzXWOGzVqlHr16lWrPTExUTExMYqLi3M8gFsXm82md999V23bttWdd9559TcCAAAAXAc8FuwjIiIUHh6uefPmyWq1Kjg4WFlZWdq/f7/S0tIc/RISErR3714dOXJEkhQUFKSgoKA6z9mlSxeFh4c7Pv/+97/XsWPHNGjQIAUGBurMmTNatWqV9u/frwULFqhVq1bX9iYBAAAAN/FYsDeZTEpLS9OSJUu0dOlSlZaWymw2KzU1VZGRkS65Ro8ePZSTk6Nt27aprKxMN910k+666y69/vrrLrsGAAAAcD0w2e12ngR1Ag/Pug8P6DQP1Nn4qHHzQJ2bB+rsHjfcw7MAAAAAXIdgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAI8G+/Pnzys5OVlDhw5VaGioYmNjlZOT49Q57Ha7EhMTZbFYlJKS0mDfvLw89ezZUxaLRaWlpVczdQAAAOC64tFgn5SUpHfffVdPPvmkVqxYIbPZrKSkJO3cubPR58jMzFRRUdEV+128eFEvvPCCOnXqdDVTBgAAAK5LHgv2O3fuVG5urpKTkzVx4kQNHjxYixcvVv/+/bVo0aJGnePUqVN69dVXNX/+/Cv2/c1vfqO2bdtq/PjxVzt1AAAA4LrjsWC/detW+fr6asSIEY42k8mkcePGqaioSEePHr3iORYuXKiBAwcqJiamwX7/+Mc/9NZbb+nFF1+Ut7f3Vc8dAAAAuN54LNgXFBTIbDbLy6vmFCwWiyQpPz+/wfHZ2dnKy8vTwoULG+x36dIlzZs3T1OmTFFoaOjVTRoAAAC4Tnks2FutVnXo0KFWe3Wb1Wqtd2xJSYlSUlI0Z84cde3atcHrrFixQmVlZXrqqaeubsIAAADAdcyj+1JMJlOTjqWkpCg4OFjx8fENnr+goEDp6elavny52rZt2+R5fldAQDuXnAeNExjo6+kpwA2os/FR4+aBOjcP1Pn65bFg7+fnV+eq/Llz5ySpztV8Sdq9e7c2b96sjIwMlZeX1zhWVVWl0tJStWnTRt7e3po/f76GDBmiAQMGOF5vWVlZKUkqKytTixYtnA78xcXlstnsTo1B0wQG+ur06TJPTwPXGHU2PmrcPFDn5oE6u4eXl6lJi8keC/Zms1lbtmyRzWarsc++em99SEhIneMKCgpks9mUkJBQ69jq1au1evVqvfHGGxo+fLiOHj2qsrIyhYWF1eobGRmpfv36KTMz00V3BAAAAHiOx4J9dHS01q5dq+3btysqKsrRnpWVpR49eshsNtc5btSoUerVq1et9sTERMXExCguLs7xAG56erouX75co9+GDRu0YcMGpaenq3Pnzi68IwAAAMBzPBbsIyIiFB4ernnz5slqtSo4OFhZWVnav3+/0tLSHP0SEhK0d+9eHTlyRJIUFBSkoKCgOs/ZpUsXhYeHOz4PHDiwVp+9e/dKkgYMGKD27du78pYAAAAAj/FYsDeZTEpLS9OSJUu0dOlSlZaWymw2KzU1VZGRkZ6aFgAAAHBDMtntdp4EdQIPz7oPD+g0D9TZ+Khx80Cdmwfq7B5NfXjWY++xBwAAAOA6BHsAAADAAAj2AAAAgAEQ7AEAAAADINgDAAAABkCwBwAAAAyAYA8AAAAYAMEeAAAAMACCPQAAAGAABHsAAADAAAj2AAAAgAEQ7AEAAAAD8Pb0BG40Xl4mT0+hWeHr3TxQZ+Ojxs0DdW4eqPO119Svsclut9tdPBcAAAAAbsZWHAAAAMAACPYAAACAARDsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAAEewAAAMAACPZwuzNnzuj5559XeHi4+vfvr6lTp+rvf/97o8d/9tlnmjZtmvr376+wsDDNmTNHp06danDM+vXrZbFYNHDgwKudPhrJHXU+ePCgFi5cqPvvv1933323hgwZounTp+vjjz929e00a+fPn1dycrKGDh2q0NBQxcbGKicnp1Fjjx8/rscff1wDBgzQ3XffrRkzZujo0aN19l25cqViYmLUp08fRUVF6Y033pDNZnPlraAB17rOx44d08svv6yxY8dqwIABCg8P19SpUxt9DVw9d/1drpaXl6eePXvKYrGotLTUFbeAKzDZ7Xa7pyeB5qOyslLjx4/XhQsX9PTTT8vPz08ZGRnKy8vT6tWr1bt37wbHFxYWasKECerbt6+mT5+uiooKLV26VHa7XRs2bFDbtm1rjTlz5ox+/OMfq2XLlrp48SKhzw3cVefFixdr7969euCBB9SzZ0+Vl5frnXfe0Z49e7Rs2TKNHDnSHbdreA8//LAOHTqkuXPnKjg4WBs2bNC7776r9PR0RURE1DuuuLhYY8aMUUBAgGbNmqUWLVro9ddf1/Hjx5WVlaWgoCBH37S0NC1fvlwzZ87UoEGD9Mknn2j58uV6+OGHNXfuXHfcZrN3rev89ttv65133tGYMWPUt29fffPNN9q4caM2bdqkn//853rooYfcdKfNlzv+Lle7ePGiHnjgAVVUVOj06dPat2+f2rdvfy1vD5JkB9zo7bfftoeEhNg/++wzR1tlZaU9MjLSPn369CuOnz17tn3IkCH28+fPO9qOHj1q79mzp33FihV1jpk1a5b90UcftT///PP2AQMGXP1N4IrcVeczZ87UGltVVWUfOXKkfdy4cVd5F7Db7fYPPvjAHhISYt+yZYujzWaz2SdPnmwfNWpUg2MXL15s79u3r/2rr75ytJWUlNjvvvtu+4IFC2q09e3b1/4///M/NcYvWbLE3rt3b/t//vMfF90N6uOOOhcXF9ttNlut8fHx8fYf/vCHLrgLNMQdNf6uRYsW2ceMGWNfsmSJPSQkxH7u3DnX3AgaxFYcuNW2bdsUEhKiu+66y9Hm4+Oj+++/X7m5uSovL6937KVLl/TBBx9o1KhRatOmjaP9jjvuUL9+/bRly5ZaY7Zs2aJdu3ZpwYIFrr0RNMhddQ4ICKg1vmXLlurZs6e++uorF91N87Z161b5+vpqxIgRjjaTyaRx48apqKiowR/Fb9u2Tffcc4+6dOniaOvYsaPuu+8+bd261dG2a9cuVVZWaty4cTXGjxs3Tt988w1bNdzAHXX29/eXyWSqNb5v376yWq26ePGii+4GdXFHjav94x//0FtvvaUXX3xR3t7err0RNIhgD7cqKChQSEhIrXaLxaLLly+rqKio3rEnTpzQxYsXdeedd9Y5vqCgoEbbuXPn9OKLL+rJJ5/UzTfffPWTR6O5s87fV1VVpU8++aTO8XBeQUGBzGazvLxq/nNhsVgkSfn5+XWOu3jxoo4fP17vfwfFxcUqLi52XMNkMtWq2W233abWrVtfsea4eu6oc13sdrvy8vLUrVs3tW7d+iruAFfirhpfunRJ8+bN05QpUxQaGurCO0BjEOzhVlarVR06dKjVXt129uzZBsd+t+93+fn56eLFizVWfBYtWqQuXbooISHhaqcNJ7mzzt/36quv6uuvv9bMmTOdnTbqcKVaVtfr+86dOye73V5vHb871mq16qabbpKPj0+tvu3bt6/3GnAdd9S5LhkZGfrss8/02GOPNWXacIK7arxixQqVlZXpqaeecsW04SR+PoImy8vLU2JiYqP67tmzR/7+/pJU549iqzV0rDF9qo/t3r1bf/nLX7R27Vq1aNGiUXNE3a7nOn/fW2+9pZUrV2rWrFkaPHjwFa+BxrmaWjam1ldzfbiOu+u8bds2vfLKK4qNjdX48eOdHg/nXesaFxQUKD09XcuXL6/zZRa49gj2aLLbb79dL7/8cqP6tmvXTtK3393XtSpw7tw5x/H6NLT6Y7Va1bp1a7Vq1UqXLl3S/PnzNXHiRN1yyy2OV2xdunRJdrtdpaWlatmypW666aZGzb25u17r/H1r1qxRSkqKHnroISUlJTVqvriyK9WyrlW86naTyVRvHavPXf2/FRUVqqqqqrVqX1paWu814DruqPN3ffDBB3rqqacUHR2t5OTkq5k6GskdNZ4/f76GDBmiAQMGOP7trayslCSVlZWpRYsWBP5rjGCPJgsMDFRsbKxTY8xmc537+I4cOaIWLVro9ttvr3ds9R7Muvbb5ufnO/bnVlRU6F//+pdWrVqlVatW1eobFham0aNHa+nSpU7Nvbm6Xuv8XX/+85+1cOFCTZ06VT//+c+dmisaZjabtWXLFtlsthp7c6vrW9e+W0lq3bq1unXrVud/B/n5+fL393c8/Gw2m2W321VQUFDjget//vOf9T5vAddyR52r7dy5U0lJSRo+fLhee+01fqrqJu6o8dGjR1VWVqawsLBafSMjI9WvXz9lZma64nZQD/bYw62io6OVn5+vw4cPO9qqqqq0adMmDR482LHiW5eWLVsqIiJC77//vioqKhztx44d04EDBxzvLG/Tpo1WrlxZ68/QoUMdx5544olrd5NwS52rrVu3zvETmvnz57v+Zpq56OholZaWavv27TXas7Ky1KNHD5nN5nrHRkVFKTc3V6dPn3a0Wa1W7dixQ9HR0Y624cOHy8fHRxs3bqwxfsOGDfL29lZkZKSL7gb1cUedpW/fgJSUlKR77rlHv/71r9WyZUvX3gjq5Y4ap6en1/q3t/ptV+np6Vq4cKGL7wrfxy+ogltVv9KusrJSzzzzjDp06KCVK1dqz549+tOf/qQ+ffo4+lb/Y/7d/xM6evSoJk6cqH79+uknP/mJ4xcXffPNN8rKymowMP7sZz/Ttm3b+AVVbuCuOr/33nt6+umnddddd2nevHm19oD279/fDXdrbHa7XdOmTdORI0f07LPPKjg4WFlZWcrKylJaWpqjfgkJCdq7d6+OHDniGHvmzBmNGTNGnTt31hNPPCFvb2+9/vrr+vLLL7Vhw4Yab6tKTU1VWlqaHnvsMYWHh+vAgQNatmyZEhIS9Pzzz7v9vpsbd9T5448/1vTp0xUYGKiXXnqp1rar3r171/kANVzDXX+Xv2/58uVKTU3lF1S5CVtx4FatWrVSRkaGXnnlFf3iF79QZWWlevfurT/84Q81wl59zGazMjIy9Nprr2n27Nny9vbWkCFD9LOf/azBUA/3cledd+7cKZvNpoMHD2ry5Mm1zvPdf5jQNCaTSWlpaVqyZImWLl2q0tJSmc1mpaamXnElvVOnTnrnnXe0ePFiPffcc7Lb7RowYIDefvvtWkHgiSeeULt27fSnP/1JK1asUOfOnTVr1izNmDHjWt4e/o876rxnzx5dvHhRJ06cqPNtZTk5OQoODnb5veFb7vq7DM9ixR4AAAAwAPbYAwAAAAZAsAcAAAAMgGAPAAAAGADBHgAAADAAgj0AAABgAAR7AAAAwAAI9gCA615CQgK/gRYAroBfUAUAzVReXp4SExPrPd6iRQsdOnTIjTMCAFwNgj0ANHP333+/hg8fXqvdy4sf6gLAjYRgDwDNXO/evTVmzBhPTwMAcJVYjgEANOjkyZOyWCxavny5srOz9cADD6hv37669957tXz5cn3zzTe1xnzxxRd64oknFB4err59+2r06NF64403dPny5Vp9T58+reTkZI0YMUJ9+vTR4MGD9fDDD2v37t21+p46dUpPP/20wsLC1L9/f02fPl3Hjh27JvcNADcaVuwBoJmrqKhQSUlJrXYfHx+1a9fO8XnHjh3KyMhQXFycOnXqpO3btys1NVX//ve/9fLLLzv6HTx4UAkJCfL29nb03bFjh1577TV98cUX+tWvfuXoe/LkSU2ZMkXFxcUaM2aM+vTpo4qKCn366afKzc3VkCFDHH0vXLig+Ph49evXT3PmzNHJkye1cuVKPf7448rOzlaLFi2u0VcIAG4MBHsAaOaWL1+u5cuX12q/9957tWLFCsfnw4cPa+3atbrrrrskSfHx8UpKStL69es1adIk9e/fX5KUkpKiqqoqrV69Wj179nT0feqpp5Sdna0JEyZo8ODBkqRf/vKX+vrrr/Xmm29q2LBhNa5vs9lqfD579qymT5+uGTNmONr8/f316quvKjc3t9Z4AGhuCPYA0MxNmjRJo0aNqtXu7+9f4/M999zjCPWSZDKZ9NOf/lTbtm3T1q1b1b9/fxUXF+uTTz5RdHS0I9RX9505c6b++te/auvWrRo8eLCsVqt27dqlYcOG1RnKv//wrpeXV623+AwaNEiS9M9//pNgD6DZI9gDQDPXvXt33XPPPVfsd8cdd9RqM5vNkqQTJ05I+nZrzXfbvz/ey8vL0ff48eOy2+3q3bt3o+bZuXNntWrVqkabn5+fJMlqtTbqHABgZDw8CwBoFJPJdMU+dru90eer7tuY80pqcA+9M9cFAKMi2AMAGuXo0aP1tnXr1q3G/9bVt6ioSDabzdGne/fuMplM/BIsAHARgj0AoFFyc3P1+eefOz7b7Xa9+eabkqSoqChJUkBAgO6++27t2LFD+fn5Nfr+7ne/kyRFR0dL+nYbzfDhw/Xhhx8qNze31vVYhQcA57DHHgCauUOHDmnjxo11HqsO7JLUs2dPTZs2TXFxcQoMDFROTo5yc3M1ZswY3X333Y5+8+bNU0JCguLi4jR16lQFBgZqx44d+uijj3T//fc73ogjSfPnz9ehQ4c0Y8YMjR07VnfddZcqKyv16aef6pZbbtGzzz577W4cAAyGYA8AzVx2drays7PrPLZlyxbH3vbIyEj16NFDK1as0LFjxxQQEKDHH39cjz/+eI0xffv21erVq7Vs2TKtWrVKFy5cULdu3TR37lz95Cc/qdG3W7duWrdunX7729/qww8/1MaNG9W+fXv17NlTkyZNujY3DAAGZbLzs04AQANOnjypESNGKCkpSbNmzfL0dAAA9WCPPQAAAGAABHsAAADAAAj2AAAAgAGwxx4AAAAwAFbsAQAAAAMg2AMAAAAGQLAHAAAADIBgDwAAABgAwR4AAAAwAII9AAAAYAD/D+AWMoFQe5SzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "# Use plot styling from seaborn.\n",
    "sns.set(style='darkgrid')\n",
    "\n",
    "# Increase the plot size and font size.\n",
    "sns.set(font_scale=1.5)\n",
    "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
    "\n",
    "# Plot the learning curve.\n",
    "plt.plot(loss_arr, 'b-o')\n",
    "\n",
    "# Label the plot.\n",
    "plt.title(\"Training loss\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_labels = np.array(sample_submission['target'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "te_labels = torch.tensor(pred_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_data = TensorDataset(te_input, te_mask, te_labels)\n",
    "prediction_sampler = SequentialSampler(prediction_data)\n",
    "prediction_dataloader = DataLoader(prediction_data, sampler = prediction_sampler, batch_size = BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting labels for 3,263 test sentences...\n",
      "    DONE.\n"
     ]
    }
   ],
   "source": [
    "print('Predicting labels for {:,} test sentences...'.format(len(te_input)))\n",
    "\n",
    "# Put model in evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# Tracking variables \n",
    "predictions , true_labels = [], []\n",
    "\n",
    "# Predict \n",
    "for batch in prediction_dataloader:\n",
    "    b_input_ids = batch[0].to(device)\n",
    "    b_input_mask = batch[1].to(device)\n",
    "    b_labels = batch[2].to(device)\n",
    "    \n",
    "    # Telling the model not to compute or store gradients, saving memory and \n",
    "    # speeding up prediction\n",
    "    with torch.no_grad():\n",
    "      # Forward pass, calculate logit predictions\n",
    "        outputs = model(b_input_ids, token_type_ids=None,\n",
    "                        attention_mask=b_input_mask)\n",
    "\n",
    "    logits = outputs[0]\n",
    "\n",
    "    # Move logits and labels to CPU\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    label_ids = b_labels.to('cpu').numpy()\n",
    "  \n",
    "    # Store predictions and true labels\n",
    "    predictions.append(logits)\n",
    "    true_labels.append(label_ids)\n",
    "\n",
    "print('    DONE.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "flat_predictions = [item for sublist in predictions for item in sublist]\n",
    "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, ..., 1, 1, 1])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flat_predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_submission['target'] = flat_predictions\n",
    "sample_submission.to_csv('submission.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
